#!/usr/bin/env python
# encoding: utf-8
__author__ = 'Zjay '
"""
@version: 3.6.3
@author: Zjay 
@license: Apache Licence 
@site: 
@software: PyCharm
@file: Spider_urllib.py
@time: 2017/12/12 16:33
"""
# region 爬取源码
"""使用request 爬取百度主页的源码"""
# import urllib.request as request
#
# # 直接爬取数据保存到本地文件
# request.urlretrieve('http://www.baidu.com',filename='D:\Project\Project_Git\Python_Spider\Spider\Comm\\baidu2.html')
# #执行完成后清除缓存
# request.urlcleanup()
#
# file = request.urlopen('http://www.baidu.com')
# date = file.read()
# dataline = file.readline
# f = open('D:\Project\Project_Git\Python_Spider\Spider\Comm\\baidu.html', 'wb')
# f.write(date)
# f.close()
# print(file.info(),file.getcode(),file.geturl())
# #print(date, ':', dataline)

# endregion

import urllib.request as request
import urllib.parse as parse

# region build_opener 添加报头
# try:
#     url = "http://blog.csdn.net/weiwei_pig/article/details/51178226"
#     # file = request.urlopen(url)
#     headers = ("User-Agent",
#                "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36")
#     opner = request.build_opener()
#     opner.addheaders = [headers]
#     data = opner.open(url).read()
#     print(data)
#     f =open('D:\Project\Project_Git\Python_Spider\Spider\Comm\\baidu3.html','wb')
#     f.write(data)
#     f.close()
# except BaseException as ex:
#     print(ex)
# finally:
#     print('End')
# endregion

# region request.Request 添加报头
# url = "http://blog.csdn.net/weiwei_pig/article/details/51178226"
# req =request.Request(url)
# req.add_header("User-Agent",
#                    "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36")
# data =request.urlopen(req).read()
# print(data)
# endregion

# region 请求超时
# for i in range(1, 10):
#     try:
#         file = request.urlopen('http://baidu.com.cn', timeout=0.1)
#         data = file.read()
#         print(data)
#     except BaseException as ex:
#         print('Error', ex)
#     finally:
#         print('End')
# endregion

# region Get 关键字
# keywd = 'hello'
# url = 'http://www.baidu.com/s?wd=' + keywd
# url = 'http://www.baidu.com/baidu?tn=monline_3_dg&ie=utf-8&wd=' + keywd
# # request.urlretrieve(url,filename='D:\Project\Project_Git\Python_Spider\Spider\Comm\\baidu4.html')
# # request.urlcleanup()
# req = request.Request(url=url)
# data = request.urlopen(req).read()
# f = open('D:\Project\Project_Git\Python_Spider\Spider\Comm\\baidu4.html', 'wb')
# f.write(data)
# f.close()

# endregion.

# region Post关键字
# url = 'http://yum.iqianyue.com/mypost/'
# postdata = parse.urlencode({'name': 'zhangsan', 'pass': 'liusi'}).encode('utf-8')
# req = request.Request(url, postdata)
# req.add_header("User-Agent",
#                "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36")
# data = request.urlopen(req).read()
# print(data)
# f = open('D:\Project\Project_Git\Python_Spider\Spider\Comm\\baidu5.html', 'wb')
# f.write(data)
# f.close()
# endregion

# region http 代理
# def use_proxy(proxy_addr, url):
#     proxy = request.ProxyHandler({'http': proxy_addr})
#     # 或者使用request.HTTPSHandler  则使用的代理类型为 Https
#     opener = request.build_opener(proxy, request.HTTPHandler)
#     request.install_opener(opener=opener)
#     data = request.urlopen(url).read().decode('utf-8')
#     return data
#
# #使用的事http  故代理类型也只能选择Http 否则无法连接
# data = use_proxy('180.156.94.218:8118', 'http://www.baidu.com')
# print(data)
# endregion

# region 启用DebugLog
# httphd = request.HTTPHandler(debuglevel=1)
# httpshd = request.HTTPSHandler(debuglevel=1)
# opener = request.build_opener(httphd, httpshd)
# request.install_opener(opener)
# data = request.urlopen('https://www.baidu.com/')
# endregion

# region URLError 异常处理
from urllib.error import URLError

try:
    #输入一个不存在的网站
    request.urlopen('http://blog.csdn.net.cn')
except  URLError as ex:
    print('Error:', ex)
    if hasattr(ex,'errno'):
        print('Error',ex.errno)
    if hasattr(ex,'strerror'):
        print (ex.strerror)
finally:
    print('End')
# endregion
